/*--
    Copyright 2003 Jan Pavlovic and Tomas Pitner.
    Masaryk University in Brno, Czech Republic
    All rights reserved.
    This code is based on the work copyrighted by
    Copyright 2001-2003 Elliotte Rusty Harold.
    All rights reserved.
    This file is part of XIncluder, a Java class library for integrating XInclude
    processing with SAX, DOM, and JDOM.
   `
   XIncluder is free software; you can redistribute it and/or modify
   it under the terms of the GNU Lesser General Public License version 2.1
   as published by the Free Software Foundation.
   XIncluder is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Lesser General Public License for more details.
   You should have received a copy of the GNU Lesser General Public License
   along with XIncluder; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESSED OR IMPLIED
   WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   DISCLAIMED.  IN NO EVENT SHALL ELLIOTTE RUSTY HAROLD OR ANY
   OTHER CONTRIBUTORS TO THIS PACKAGE
   BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   SUCH DAMAGE.
   */
  package net.sf.tomp.xml.include;
  
  import net.sf.tomp.xtcl.filter.XTFilterImpl;
  
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.xml.sax.Attributes;
  import org.xml.sax.EntityResolver;
  import org.xml.sax.Locator;
  import org.xml.sax.SAXException;
  import org.xml.sax.XMLReader;
  import org.xml.sax.helpers.AttributesImpl;
  import org.xml.sax.helpers.NamespaceSupport;
  import org.xml.sax.helpers.XMLReaderFactory;
  
  import java.io.BufferedInputStream;
  import java.io.BufferedReader;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.InputStreamReader;
  import java.io.UnsupportedEncodingException;
  
  import java.net.MalformedURLException;
  import java.net.URL;
  import java.net.URLConnection;
  
  import java.util.Stack;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  /***
   * Copyright 2003 Jan Pavlovic and Tomas Pitner. Masaryk University in Brno,
   * Czech Republic All rights reserved. This code is based on the work
   * copyrighted by Copyright 2001-2003 Elliotte Rusty Harold. All rights
   * reserved.
   * <p>
   * This is a SAX filter which resolves all XInclude include elements before
   * passing them on to the client application. Currently this class has the
   * following known deviation from the XInclude specification:
   * </p>
   * <ol>
   * <li>XPointer is not supported.</li>
   * </ol>
   * <p>
   * Extensions made by JP and TP: <br/ > the URL of the included TEXT document
   * (i.e. if <code>parse='text'</code>) can be in one of the following forms
   * (lines are numbered starting at 1):
   * <ul>
   * <li><code>textfileurl#startLineNumber</code> will include only the line
   * with startLineNumber <br/></li>
   * <li><code>textfileurl#startLineNumber$count</code> will include count
   * lines beginning from startLineNumber <br/></li>
   * <li><code>textfileurl#startLineNumber-endLineNumber</code> will include
   * all lines beginning from startLineNumber and ending to (including) line
   * endLineNumber <br/></li>
   * <li><code>textfileurl#startLineNumber-</code> will include all lines
   * beginning from to the end of file <br/></li>
   * <li><code>textfileurl#$countOfLines</code> or
   * <code>textfileurl#-countOfLines</code> will include first countOfLines
   * lines from the file <br/></li>
   * </ul>
   * TO-DO: check whether the xml:base-s really work for file: URLs <br/ > TO-DO:
   * check whether the xml:base-s really work for file: URLs <br/>
   * </p>
   * <p>
  * Extensions made by TP, Nov 2003: <br/ > Instead of specifying and start/end
  * line and/or line count nummericaly, Java-style regexp patterns can be put in
  * slashes instead of the first or last line number.
  * <ul>
  * <li><code>textfileurl#/startLineRegexp/</code> will include only the line
  * with startLineRegexp <br/></li>
  * <li><code>textfileurl#/startLineRegexp/$count</code> will include count
  * lines beginning from start line <br/></li>
  * <li><code>textfileurl#/startLineRegexp/-/endLineRegexp/</code> will
  * include all lines beginning from start line and ending to line (including)
  * matching the end pattern <br/></li>
  * <li><code>textfileurl#/startLineRegexp/-endLineNo</code> will include all
  * lines beginning from start line and ending to line (including) specified by
  * <code>endLineNo</code> <br/></li>
  * <li><code>textfileurl#startLineNo-/endLineRegexp/</code> will include all
  * lines beginning from the start line number and ending to line (including)
  * matching the end pattern <br/></li>
  * <li><code>textfileurl#-/endLineRegexp/</code> will include all lines upto
  * the line (including) matching the end pattern <br/></li>
  * <li><code>textfileurl#/startLineRegexp/-</code> will include all lines
  * beginning from to the end of file <br/></li>
  * </ul>
  * </p>
  * <p>
  * Furthermore, I would definitely use a new instance of this class for each
  * document you want to process. I doubt it can be used successfully on multiple
  * documents. Furthermore, I can virtually guarantee that this class is not
  * thread safe. You have been warned.
  * </p>
  * <p>
  * Since this class is not designed to be subclassed, and since I have not yet
  * considered how that might affect the methods herein or what other protected
  * methods might be needed to support subclasses, I have declared this class
  * final. I may remove this restriction later, though the use-case for
  * subclassing is weak. This class is designed to have its functionality
  * extended via a a horizontal chain of filters, not a vertical hierarchy of sub
  * and superclasses.
  * </p>
  * <p>
  * To use this class:
  * </p>
  * <ol>
  * <li>Construct an <code>XIncludeFilter</code> object with a known base URL
  * </li>
  * <li>Pass the <code>XMLReader</code> object from which the raw document
  * will be read to the <code>setParent()</code> method of this object.</li>
  * <li>Pass your own <code>ContentHandler</code> object to the
  * <code>setContentHandler()</code> method of this object. This is the object
  * which will receive events from the parsed and included document.</li>
  * <li>Optional: if you wish to receive comments, set your own
  * <code>LexicalHandler</code> object as the value of this object's
  * http://xml.org/sax/properties/lexical-handler property. Also make sure your
  * <code>LexicalHandler</code> asks this object for the status of each comment
  * using <code>insideIncludeElement</code> before doing anything with the
  * comment.</li>
  * <li>Pass the URL of the document to read to this object's
  * <code>parse()</code> method</li>
  * </ol>
  * <p>
  * e.g.
  * </p>
  * 
  * <pre><code>
  * XIncludeFilter includer = new XIncludeFilter(base);
  * includer.setParent(parser);
  * includer.setContentHandler(new SAXXIncluder(System.out));
  * includer.parse(args[i]);
  * </code>
  </pre>
  * 
  * @author Elliotte Rusty Harold, Tomas Pitner, Jan Pavlovic
  * @version 1.0, April 7, 2004
  */
 public class XIncludeFilter extends XTFilterImpl {
     /***
      * the delimiters for regular expressions used to specify the first or last
      * line
      */
     protected static final String REGEXP_DELIMITERS = "/";
 
     /***
      * the number of the first line to be included is specified in the URL after
      * the <code>term1</code> String
      */
     protected static final String term1 = "#";
 
     /***
      * the number of lines to be included is specified in the URL after the
      * <code>term2</code> String
      */
     protected static final String term2 = "$";
 
     /***
      * the number of the last line to be included is specified in the URL after
      * the <code>term3</code> String
      */
     protected static final String term3 = "-";
 
     private static final String XINCLUDE_NAMESPACE = "http://www.w3.org/2001/XInclude";
 
     private Log log = LogFactory.getLog(AdaptiveXIncludeFilter.class);
 
     private Stack bases = new Stack();
 
     private Stack locators = new Stack();
 
     // necessary to throw away contents of non-empty XInclude elements
     private int level = 0;
 
     private int depth = 0;
 
     private boolean atRoot = false;
 
     public String getXIncludeNamespace() {
         return XINCLUDE_NAMESPACE;
     }
 
     /*
      * private EntityResolver resolver; public XIncludeFilter() { this(null); }
      * public XIncludeFilter(EntityResolver resolver) { this.resolver =
      * resolver; }
      */
 
     // what if this isn't called????
     // do I need to check this in startDocument() and push something
     // there????
     public void setDocumentLocator(Locator locator) {
         //System.out.println("+++ADAPTIVEXINCLUDER.setDocumentLocator("+locator);
         //System.out.println("+++ADAPTIVEXINCLUDER.documentLocator.getSystemId="+locator.getSystemId());
         //if(locator == null || locator.getSystemId() == null) return;
         super.setDocumentLocator(locator);
 
         locators.push(locator);
 
         if (locator == null) {
             return;
         }
 
         String base = locator.getSystemId();
 
         try {
             // for some buggy :-) xmlreaders (or emmiters),
             // locator.getSystemId() is null,
             // and causes problems with bases.push(new URL(null))!!!
             if (base != null) {
                 bases.push(new URL(base));
 
                 //System.err.println(">>> setDocumentLocator:pushing
                 // base="+base+" atRoot="+atRoot+" level="+level);
             } else {
                 // FIXME: workaround for null locator.getSystemId() - some base
                 // must be pushed...
                 //if(bases.size() == 0)
                 //bases.push(new URL("file:my-url.txt"));
                 log.debug(">>> " + this
                         + "setDocumentLocator: trying to push base null in "
                         + locator + ", sysId=" + locator.getSystemId());
 
                 //System.out.println(">>> setDocumentLocator:pushing base="+new
                 // URL("file:my-url.txt")+" bases.size()="+bases.size());
                 //else
                 //   bases.push(bases.peek());
             }
         } catch (MalformedURLException e) {
             e.printStackTrace();
             throw new UnsupportedOperationException("Unrecognized SYSTEM ID: "
                     + base);
         }
     }
 
     /***
      * <p>
      * This utility method returns true if and only if this reader is currently
      * inside a non-empty include element. (This is <strong>not </strong> the
      * same as being inside the node set which replaces the include element.)
      * This is primarily needed for comments inside include elements. It must be
      * checked by the actual LexicalHandler to see whether a comment is passed
      * or not.
      * </p>
      * 
      * @return boolean
      */
     public boolean insideIncludeElement() {
         return level != 0;
     }
 
     public void startElement(String uri, String localName, String qName,
             Attributes atts) throws SAXException {
         //System.out.println("ADAPTIVEXINCLUDER.startElement,
         // localname="+localName+" uri="+uri);
         if (level == 0) { // We're not inside an xi:include element
 
             String base = atts.getValue(NamespaceSupport.XMLNS, "base");
             URL parentBase = (URL) bases.peek();
             URL currentBase = parentBase;
 
             if (base != null) {
                 try {
                     currentBase = new URL(parentBase, base);
                 } catch (MalformedURLException e) {
                     throw new SAXException(
                             "Malformed base URL: " + currentBase, e);
                 }
             }
 
             bases.push(currentBase); //System.out.println(">>> startElement
 
             if (uri.equals(getXIncludeNamespace())
                     && localName.equals("include")) {
                 // include external document
                 String href = atts.getValue("href");
 
                 // Verify that there is an href attribute
                 if (href == null) {
                     throw new SAXException("Missing href attribute");
                 }
 
                 String parse = atts.getValue("parse");
 
                 if (parse == null) {
                     parse = "xml";
                 }
 
                 if (parse.equals("text")) {
                     String encoding = atts.getValue("encoding");
 
                     includeTextDocument(href, encoding);
                 } else if (parse.equals("xml")) {
                     String variant = atts.getValue("variant");
 
                     includeXMLDocument(href, variant);
                 }
 
                 // Need to check this also in DOM and JDOM????
                 else {
                     throw new SAXException(
                             "Illegal value for parse attribute: " + parse);
                 }
 
                 level++;
             } else {
                 if (atRoot) {
                     // add xml:base attribute if necessary
                     AttributesImpl attsImpl = new AttributesImpl(atts);
 
                     attsImpl.addAttribute(NamespaceSupport.XMLNS, "base",
                             "xml:base", "CDATA", currentBase.toExternalForm());
                     atts = attsImpl;
                     atRoot = false;
                 }
 
                 super.startElement(uri, localName, qName, atts);
             }
         }
     }
 
     public void endElement(String uri, String localName, String qName)
             throws SAXException {
         if (uri.equals(getXIncludeNamespace()) && localName.equals("include")) {
             level--;
         } else if (level == 0) {
             bases.pop();
             super.endElement(uri, localName, qName);
         }
 
         //System.out.println("+++ADAPTIVEXINCLUDER.endElement,
         // localname="+localName);
     }
 
     public void startDocument() throws SAXException {
         //System.out.println("+++ADAPTIVEXINCLUDER.startDocument");
         level = 0;
 
         if (depth == 0) {
             super.startDocument();
         }
 
         depth++;
     }
 
     public void endDocument() throws SAXException {
         depth--;
 
         if (depth == 0) {
             //System.out.println("+++trying to do
             // ADAPTIVEXINCLUDER.endDocument");
             try {
                 //System.out.println("++++depth="+depth+"++++
                 // "+this+".endDocument: locators#"+locators.size()+"
                 // .pop()="+locators);
                 locators.pop();
             } catch (java.util.EmptyStackException ese) {
                 log
                         .error(
                                 "+++ADAPTIVEXINCLUDER.endDocument caused Empty stack in locators",
                                 ese);
             }
 
             try {
                 Object p = bases.pop(); // pop the URL for the document itself
             } catch (java.util.EmptyStackException ese) {
                 throw ese;
 
                 //System.out.println("+++ADAPTIVEXINCLUDER.endDocument caused
                 // Empty stack in bases");
             }
 
             //System.out.println("+++alive in ADAPTIVEXINCLUDER.endDocument");
             super.endDocument();
         }
 
         //System.out.println("+++ADAPTIVEXINCLUDER.endDocument");
     }
 
     // how do prefix mappings move across documents????
     public void startPrefixMapping(String prefix, String uri)
             throws SAXException {
         if (level == 0) {
             super.startPrefixMapping(prefix, uri);
         }
     }
 
     public void endPrefixMapping(String prefix) throws SAXException {
         if (level == 0) {
             super.endPrefixMapping(prefix);
         }
     }
 
     public void characters(char[] ch, int start, int length)
             throws SAXException {
         // System.out.println("characters: "+new String(ch, start, length));
         if (level == 0) {
             super.characters(ch, start, length);
         }
 
         //System.out.println("+++ADAPTIVEXINCLUDER.characters="+new String(ch,
         // start, length));
     }
 
     public void ignorableWhitespace(char[] ch, int start, int length)
             throws SAXException {
         if (level == 0) {
             super.ignorableWhitespace(ch, start, length);
         }
     }
 
     public void processingInstruction(String target, String data)
             throws SAXException {
         if (level == 0) {
             super.processingInstruction(target, data);
         }
     }
 
     public void skippedEntity(String name) throws SAXException {
         if (level == 0) {
             super.skippedEntity(name);
         }
     }
 
     // convenience method for error messages
     private String getLocation() {
         String locationString = "";
         Locator locator = (Locator) locators.peek();
         String publicID = "";
         String systemID = "";
         int column = -1;
         int line = -1;
 
         if (locator != null) {
             publicID = locator.getPublicId();
             systemID = locator.getSystemId();
             line = locator.getLineNumber();
             column = locator.getColumnNumber();
         }
 
         locationString = " in document included from " + publicID + " at "
                 + systemID + " at line " + line + ", column " + column;
 
         return locationString;
     }
 
     /***
      * <p>
      * This method reads URL and returns the regular expression to match on the
      * first line.
      * </p>
      * 
      * @param url URL of the document that will be read
      * @return the regular expression to match on the first line.
      */
     protected static String getFirstLineRegexp(String url) {
         int indexTerm1 = url.indexOf(term1) + 1; // position after #
 
         if (indexTerm1 == 0) { // if no # return 0
 
             return null;
         }
 
         int separatorIndex = url.indexOf(term2); // from # until $
 
         if (separatorIndex < 0) {
             separatorIndex = url.indexOf(term3); // from # until -
         }
 
         if (separatorIndex < 0) {
             separatorIndex = url.length(); // take whole URL
         }
 
         if (indexTerm1 == separatorIndex) { // if #$ or #- return ""
 
             return null;
         }
 
         // trim also the leading slashes
         // http://something.com/somepage#/firstlineregexp/-/lastlineregexp/
         // must give "firstlineregexp"
         char begin = url.charAt(indexTerm1);
 
         if (REGEXP_DELIMITERS.indexOf(begin) >= 0) {
             return url.substring(indexTerm1 + 1, separatorIndex - 1);
         } else {
             return null;
         }
     }
 
     /***
      * <p>
      * This method reads URL and returns the regular expression to match on the
      * last line.
      * </p>
      * 
      * @param url URL of the document that will be read
      * @return the regular expression to match on the last line.
      */
     protected static String getLastLineRegexp(String url) {
         // if no $ - return null
         if (url.indexOf(term1) == -1) {
             //System.err.println("url.indexOf(term1) == -1: no $ -");
             return null;
         }
 
         int indexTerm2 = url.indexOf(term2) + 1;
         int indexTerm3 = url.indexOf(term3) + 1;
 
         //System.err.println("line-count: indexTerm2="+indexTerm2+",
         // indexTerm3="+indexTerm3+", url.length="+url.length());
         // if no $ neither - then include one line:
         if ((indexTerm2 == 0) && (indexTerm3 == 0)) {
             //System.err.println("indexTerm2 == 0 && indexTerm3 == 0 because no
             // $ neither - ");
             return null;
         }
 
         // if - at the end of the url then include all lines to the end:
         if (indexTerm3 == url.length()) {
             //System.err.println("indexTerm3 == url.length() - at the end");
             return null;
         }
 
         String count = url.substring(indexTerm2).trim(); // from $ to end
         String last = url.substring(indexTerm3).trim(); // from - to end
 
         // if '$' (count of lines) specified - no regexp returned!
         if (indexTerm2 > 0) {
             // System.err.println("line-count: parsing count='"+count+"'");
             //System.err.println("indexTerm2 > 0 because $ present");
             return null;
 
             // if '-' (last line number) specified
             // - return the string last with leading/trailing slashes trimmed
         } else {
             // System.err.println("line-count: parsing last='"+last+"'");
             char begin = last.charAt(last.length() - 1);
 
             if (REGEXP_DELIMITERS.indexOf(begin) >= 0) {
                 return last.substring(1, last.length() - 1);
             } else {
                 //System.err.println("REGEXP_DELIMITERS.indexOf(begin) < 0 - no
                 // regexp delimiter found ");
                 return null;
             }
         }
     }
 
     /***
      * <p>
      * This method reads URL and return the first line to read
      * </p>
      * 
      * @param url URL of the document that will be read
      * @return int
      */
     protected int getLineBegin(String url) {
         int indexTerm1 = url.indexOf(term1) + 1; // position after #
 
         if (indexTerm1 == 0) { // if no # return 0
 
             return 0;
         }
 
         int separatorIndex = url.indexOf(term2); // from # until $
 
         if (separatorIndex < 0) {
             separatorIndex = url.indexOf(term3); // from # until -
         }
 
         if (separatorIndex < 0) {
             separatorIndex = url.length(); // take whole URL
         }
 
         if (indexTerm1 == separatorIndex) { // if #$ or #- return 1
 
             return 1;
         }
 
         String begin = url.substring(indexTerm1, separatorIndex).trim();
 
         try {
             // System.err.println("line-begin: parsing begin='"+begin+"'");
             return Integer.parseInt(begin);
         } catch (NumberFormatException nfe) {
             return 0;
         }
     }
 
     /***
      * <p>
      * This method reads URL and return the amount of line line to read
      * </p>
      * 
      * @param url URL of the document that will be read
      * @return int
      */
     protected int getLineCount(String url) {
         if (url.indexOf(term1) == -1) {
             return Integer.MAX_VALUE;
         }
 
         int indexTerm2 = url.indexOf(term2) + 1;
         int indexTerm3 = url.indexOf(term3) + 1;
 
         //System.err.println("line-count: indexTerm2="+indexTerm2+",
         // indexTerm3="+indexTerm3+", url.length="+url.length());
         // if no $ neither - then include one line:
         if ((indexTerm2 == 0) && (indexTerm3 == 0)) {
             return 1;
         }
 
         // if - at the end of the url then include all lines to the end:
         if (indexTerm3 == url.length()) {
             return Integer.MAX_VALUE;
         }
 
         String count = url.substring(indexTerm2).trim(); // from $ to end
         String last = url.substring(indexTerm3).trim(); // from - to end
 
         try {
             // if '$' (count of lines) specified
             if (indexTerm2 > 0) {
                 // System.err.println("line-count: parsing count='"+count+"'");
                 int iCount = Integer.parseInt(count);
 
                 return iCount;
 
                 // if '-' (last line number) specified
             } else {
                 // System.err.println("line-count: parsing last='"+last+"'");
                 int iLast = Integer.parseInt(last);
 
                 //System.err.println("line-count returning iLast + 1 -
                 // getLineBegin(url)=" +(iLast + 1 - getLineBegin(url)));
                 return (iLast + 1) - getLineBegin(url);
             }
         } catch (NumberFormatException nfe) {
             return 0;
         }
     }
 
     protected boolean isLastLineNo(String url) {
         if (url.indexOf(term1) == -1) {
             return false;
         }
 
         int indexTerm2 = url.indexOf(term2) + 1; // $
         int indexTerm3 = url.indexOf(term3) + 1; // -
 
         if ((indexTerm2 == 0) && (indexTerm3 == 0)) {
             return false;
         }
 
         // if - at the end of the url then include all lines to the end:
         if (indexTerm3 == url.length()) {
             return false;
         }
 
         String count = url.substring(indexTerm2).trim(); // from $ to end
         String last = url.substring(indexTerm3).trim(); // from - to end
 
         try {
             // if '$' (count of lines) specified
             if (indexTerm2 > 0) {
                 return false;
 
                 // if '-' (last line number) specified
             } else {
                 // System.err.println("line-count: parsing last='"+last+"'");
                 int iLast = Integer.parseInt(last);
 
                 return true;
             }
         } catch (NumberFormatException nfe) {
             return false;
         }
     }
 
     /***
      * <p>
      * This utility method reads a document at a specified URL and fires off
      * calls to <code>characters()</code>. It's used to include files with
      * <code>parse="text"</code>
      * </p>
      * 
      * @param url URL of the document that will be read
      * @param encoding Encoding of the document; e.g. UTF-8, ISO-8859-1, etc.
      * @throws SAXException if the requested document cannot be downloaded from
      *             the specified URL or if the encoding is not recognized
      */
     private void includeTextDocument(String url, String encoding)
             throws SAXException {
         String firstLineRegexp = getFirstLineRegexp(url);
         String lastLineRegexp = getLastLineRegexp(url);
 
         int begin = getLineBegin(url);
         int count = getLineCount(url);
 
         boolean isLastLineNo = isLastLineNo(url);
 
         // System.err.println("include-text: begin="+begin+" count="+count);
         // if count=0, i.e. include no lines, then return immediately
         if (((count == 0) && (lastLineRegexp == null))
                 || (isLastLineNo && (count == 1))) {
             //System.err.println("count == 0");
             return;
         }
 
         if ((encoding == null) || encoding.trim().equals("")) {
             encoding = "UTF-8";
         }
 
         URL source;
 
         try {
             URL base = (URL) bases.peek();
 
             source = new URL(base, url);
         } catch (MalformedURLException e) {
             UnavailableResourceException ex = new UnavailableResourceException(
                     "Unresolvable URL " + url + getLocation());
 
             ex.setRootCause(e);
             throw new SAXException("Unresolvable URL " + url + getLocation(),
                     ex);
         }
 
         try {
             URLConnection uc = source.openConnection();
             InputStream in = new BufferedInputStream(uc.getInputStream());
             String encodingFromHeader = uc.getContentEncoding();
             String contentType = uc.getContentType();
 
             if (encodingFromHeader != null) {
                 encoding = encodingFromHeader;
             } else {
                 // What if file does not have a MIME type but name ends in
                 // .xml????
                 // MIME types are case-insensitive
                 // Java may be picking this up from file URL
                 if (contentType != null) {
                     contentType = contentType.toLowerCase();
 
                     if (contentType.equals("text/xml")
                             || contentType.equals("application/xml")
                             || (contentType.startsWith("text/") && contentType
                                     .endsWith("+xml"))
                             || (contentType.startsWith("application/") && contentType
                                     .endsWith("+xml"))) {
                         encoding = EncodingHeuristics
                                 .readEncodingFromStream(in);
                     }
                 }
             }
 
             //System.err.println("dealing with begin="+begin);
             //System.err.println("dealing with count="+count);
             if ((begin != 0) || (firstLineRegexp != null)) {
                 BufferedReader br = new BufferedReader(new InputStreamReader(
                         in, encoding));
                 String ls = System.getProperty("line.separator");
 
                 if (begin != 0) {
                     // first line number specified
                     String line = br.readLine();
 
                     while (line != null) {
                         String newLine = br.readLine();
 
                         if ((--begin <= 0) && (--count >= 0)) {
                             if ((count > 0) && (newLine != null)) {
                                 line = line + ls;
                             }
 
                             this.characters(line.toCharArray(), 0, line
                                     .length());
                         }
 
                         line = newLine;
                     }
                 } else {
                     // firstLineRegexp != null
                     //System.err.println("dealing with
                     // firstLineRegexp="+firstLineRegexp);
                     //System.err.println("dealing with
                     // lastLineRegexp="+lastLineRegexp);
                     boolean wasFirst = false;
                     boolean beforeLast = true;
 
                     Pattern firstLinePattern = (firstLineRegexp == null) ? null
                             : Pattern.compile(firstLineRegexp);
                     Pattern lastLinePattern = (lastLineRegexp == null) ? null
                             : Pattern.compile(lastLineRegexp);
 
                     if (firstLinePattern != null) {
                         begin = Integer.MAX_VALUE;
                     }
 
                     if (lastLinePattern != null) {
                         count = Integer.MAX_VALUE;
                     }
 
                     String line = br.readLine();
 
                     boolean testLastLineNo = (lastLinePattern == null)
                             && isLastLineNo;
                     int lastLineNo = count - 1;
 
                     while ((line != null) && beforeLast) {
                         String newLine = br.readLine();
 
                         //System.err.println("\nline "+lastLineNo+".="+line);
                         Matcher firstLineMatcher = (firstLinePattern == null) ? null
                                 : firstLinePattern.matcher(line);
                         Matcher lastLineMatcher = (lastLinePattern == null) ? null
                                 : lastLinePattern.matcher(line);
 
                         boolean firstLineMatched = (firstLineMatcher == null) ? false
                                 : firstLineMatcher.find();
                         boolean lastLineMatched = (lastLineMatcher == null) ? false
                                 : lastLineMatcher.find();
 
                         //System.err.println("firstLineMatcher.find()="+firstLineMatched);
                         //System.err.println("lastLineMatcher.find()="+lastLineMatched);
                         wasFirst |= ((--begin <= 0) || ((firstLineMatcher != null) && firstLineMatched));
 
                         /*
                          * if( --count < 0) { System.err.println("falsing
                          * beforeLast - count low"); beforeLast = false; }
                          */
                         if (wasFirst && beforeLast) {
                             //if ((count > 0 || (lastLineMatcher != null &&
                             // !lastLineMatched))
                             if ((count > 0) && !lastLineMatched
                                     && (!testLastLineNo || (lastLineNo > 1))
                                     && (newLine != null)) {
                                 line = line + ls;
                             }
 
                             this.characters(line.toCharArray(), 0, line
                                     .length());
                             count--;
                         }
 
                         if ((count <= 0) || lastLineMatched
                                 || (testLastLineNo && (lastLineNo <= 1))) {
                             //System.err.println("falsing beforeLast
                             // count="+count+" lastLineMatched="
                             //    +lastLineMatched+"
                             // testLastLineNo="+testLastLineNo);
                             beforeLast = false;
                         }
 
                         line = newLine;
                         lastLineNo--;
                     }
                 }
             } else {
                 InputStreamReader reader = new InputStreamReader(in, encoding);
                 char[] c = new char[1024];
 
                 while (true) {
                     int charsRead = reader.read(c, 0, 1024);
 
                     if (charsRead == -1) {
                         break;
                     }
 
                     if (charsRead > 0) {
                         this.characters(c, 0, charsRead);
                     }
                 }
             }
         } catch (UnsupportedEncodingException e) {
             throw new SAXException("Unsupported encoding: " + encoding
                     + getLocation(), e);
         } catch (IOException e) {
             throw new SAXException("Document not found: "
                     + source.toExternalForm() + getLocation(), e);
         }
     }
 
     /***
      * <p>
      * This utility method reads a document at a specified URL and fires off
      * calls to various <code>ContentHandler</code> methods. It's used to
      * include files with <code>parse="xml"</code>
      * </p>
      * 
      * @param url URL of the document that will be read
      * @param variant DOCUMENT ME!
      * @throws SAXException if the requested document cannot be downloaded from
      *             the specified URL.
      */
     protected void includeXMLDocument(String url, String variant)
             throws SAXException {
         log.info("XI:INCLUDE href=" + url + " in " + this);
         log.debug("bases=" + bases);
 
         URL source;
 
         try {
             URL base = (URL) bases.peek();
 
             source = new URL(base, url);
         } catch (MalformedURLException e) {
             UnavailableResourceException ex = new UnavailableResourceException(
                     "Unresolvable URL " + url + getLocation());
 
             ex.setRootCause(e);
             throw new SAXException("Unresolvable URL " + url + getLocation(),
                     ex);
         }
 
         try {
             // make this more robust
             XMLReader parser = getXMLReader(variant);
 
             if (parser == null) {
                 return;
             }
 
             parser.setContentHandler(this);
 
             EntityResolver resolver = this.getEntityResolver();
 
             if (resolver != null) {
                 parser.setEntityResolver(resolver);
             }
 
             // also other handlers need to be set!
             parser.setDTDHandler(this);
             parser.setEntityResolver(this);
             parser.setErrorHandler(this);
 
             try {
                 parser.setProperty(
                         "http://xml.org/sax/properties/lexical-handler", this);
             } catch (SAXException se) {
                 se.printStackTrace();
             }
 
             // save old level and base
             int previousLevel = level;
 
             this.level = 0;
 
             if (bases.contains(source)) {
                 Exception e = new CircularIncludeException(
                         "Circular XInclude Reference to " + source
                                 + getLocation() + ", bases=" + bases);
 
                 // FIXME: does not test circularities!
                 throw new SAXException("Circular XInclude Reference", e);
             }
 
             bases.push(source); //System.err.println(">>> pushing included to
 
             // base="+source);
             atRoot = true;
 
             //System.out.println("%%%%%%%%%%%%% XIncludeFilter will parse
             // "+source.toExternalForm());
             // source.toExternalForm()
             //Source src = new StreamSource(source.toString());
             //System.out.println("%%%%%%%%%%%%% will parse="+source);
             parser.parse(source.toExternalForm());
 
             // restore old level and base
             this.level = previousLevel;
 
             /* URL popped = (URL) */bases.pop(); //System.err.println("<<<
 
             // popping base="+popped);
         } catch (IOException e) {
             throw new SAXException("Document not found: "
                     + source.toExternalForm() + getLocation(), e);
         }
 
         //System.out.println("...XI:INCLUDED");
     }
 
     public XMLReader getXMLReader(String variant) throws SAXException {
         XMLReader parser = null;
 
         try {
             parser = XMLReaderFactory.createXMLReader();
         } catch (SAXException e) {
             parser = XMLReaderFactory.createXMLReader(
 
             //"org.apache.xerces.parsers.SAXParser"
                     "org.apache.crimson.parser.XMLReaderImpl");
         }
 
         return parser;
     }
 
     /*
      * public static void main(String args[]) throws Exception {
      * //System.out.println("firstlineregexp="+getFirstLineRegexp(args[0]));
      * //System.out.println("lastlineregexp="+getLastLineRegexp(args[0]));
      * XIncludeFilter f = new XIncludeFilter(); f.bases.push(new URL("file:."));
      * f.includeTextDocument(args[0], "Windows-1250"); }
      */
 }